{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "【项目01】  商铺数据加载及存储\n",
    "\n",
    "作业要求：\n",
    "1、成功读取“商铺数据.csv”文件\n",
    "2、解析数据，存成列表字典格式：[{'var1':value1,'var2':value2,'var3':values,...},...,{}]\n",
    "3、数据清洗：\n",
    "① comment，price两个字段清洗成数字\n",
    "② 清除字段缺失的数据\n",
    "③ commentlist拆分成三个字段，并且清洗成数字\n",
    "4、结果存为.pkl文件\n",
    "\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\\ufeffclassify', 'name', 'comment', 'star', 'price', 'address', 'commentlist\\n']\n",
      "['美食', '望蓉城老坛酸菜鱼(合生汇店)', '我要点评', '该商户暂无星级', '人均                                        ￥125', '翔殷路1099号合生汇5F', '口味8.3                                环境8.4                                服务8.5\\n']\n",
      "['美食', '泰国街边料理', '74                    条点评', '准四星商户', '人均                                        ￥48', '黄兴路合生汇B2美食集市内', '口味7.4                                环境7.6                                服务7.4\\n']\n",
      "['美食', '壹面如故(苏宁生活广场店)', '265                    条点评', '准四星商户', '人均                                        ￥21', '邯郸路585号苏宁生活广场B1层', '口味7.0                                环境7.2                                服务7.2\\n']\n",
      "['美食', '鮨谷•Aburiya(合生汇店)', '2748                    条点评', '准五星商户', '人均                                        ￥142', '翔殷路1099号合生广场5楼23、28铺位', '口味8.9                                环境8.5                                服务8.4\\n']\n"
     ]
    }
   ],
   "source": [
    "# 数据读取\n",
    "\n",
    "f = open('C:/Users/Hjx/Desktop/商铺数据.csv','r',encoding='utf8')\n",
    "for i in f.readlines()[:5]:\n",
    "    print(i.split(','))  \n",
    "# 查看数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 创建comment、price、commentlist清洗函数\n",
    "# 函数式编程\n",
    "\n",
    "def fcm(s):\n",
    "    if '条' in s:\n",
    "        return int(s.split(' ')[0])\n",
    "    else:\n",
    "        return '缺失数据'\n",
    "# comment清洗函数：用空格分段，选取结果list的第一个为点评数，并且转化为整型\n",
    "\n",
    "def fpr(s):\n",
    "    if '￥' in s:\n",
    "        return float(s.split('￥')[-1])\n",
    "    else:\n",
    "        return  '缺失数据'\n",
    "# print清洗函数：用￥分段，选取结果list的最后一个为人均价格，并且转化为浮点型\n",
    "\n",
    "def fcl(s):\n",
    "    if ' ' in s:\n",
    "        quality = float(s.split('                                ')[0][2:])\n",
    "        environment = float(s.split('                                ')[1][2:])\n",
    "        service = float(s.split('                                ')[2][2:-1])\n",
    "        return [quality,environment,service]\n",
    "    else:\n",
    "        return  '缺失数据'\n",
    "# commentlist清洗函数：用空格分段，分别清洗出质量、环境及服务数据，并转化为浮点型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "成功加载1条数据\n",
      "成功加载2条数据\n",
      "成功加载3条数据\n",
      "成功加载4条数据\n",
      "成功加载5条数据\n",
      "成功加载6条数据\n",
      "成功加载7条数据\n",
      "成功加载8条数据\n",
      "成功加载9条数据\n",
      "成功加载10条数据\n",
      "成功加载11条数据\n",
      "成功加载12条数据\n",
      "成功加载13条数据\n",
      "成功加载14条数据\n",
      "成功加载15条数据\n",
      "[{'comment_count': 74, 'classify': '美食', 'environment': 0.6, 'price': 48.0, 'address': '黄兴路合生汇B2美食集市内', 'star': '准四星商户', 'name': '泰国街边料理', 'quality': 0.4, 'service': 0.4}, {'comment_count': 265, 'classify': '美食', 'environment': 0.2, 'price': 21.0, 'address': '邯郸路585号苏宁生活广场B1层', 'star': '准四星商户', 'name': '壹面如故(苏宁生活广场店)', 'quality': 0.0, 'service': 0.2}, {'comment_count': 2748, 'classify': '美食', 'environment': 0.5, 'price': 142.0, 'address': '翔殷路1099号合生广场5楼23、28铺位', 'star': '准五星商户', 'name': '鮨谷•Aburiya(合生汇店)', 'quality': 0.9, 'service': 0.4}, {'comment_count': 785, 'classify': '美食', 'environment': 0.2, 'price': 24.0, 'address': '邯郸路600号万达商业广场B1楼A05号铺', 'star': '准四星商户', 'name': '麦当劳(万达店)', 'quality': 0.4, 'service': 0.2}, {'comment_count': 3779, 'classify': '美食', 'environment': 0.6, 'price': 70.0, 'address': '淞沪路8号百联又一城购物中心7层', 'star': '准五星商户', 'name': '蒸年青STEAMYOUNG(百联又一城购物中心店)', 'quality': 0.6, 'service': 0.6}, {'comment_count': 458, 'classify': '美食', 'environment': 0.5, 'price': 14.0, 'address': '淞沪路8号百联又一城购物中心B1层', 'star': '准四星商户', 'name': '丸来玩趣(百联又一城购物中心店)', 'quality': 0.0, 'service': 0.5}, {'comment_count': 1280, 'classify': '美食', 'environment': 0.5, 'price': 47.0, 'address': '翔殷路1099号合生汇4F', 'star': '四星商户', 'name': '韩国雪冰(合生汇店)', 'quality': 0.7, 'service': 0.6}, {'comment_count': 85, 'classify': '美食', 'environment': 0.9, 'price': 81.0, 'address': '邯郸路600号万达商业广场特力时尚汇3层', 'star': '准五星商户', 'name': '小菜园新徽菜(五角场万达广场店)', 'quality': 0.5, 'service': 0.0}, {'comment_count': 30, 'classify': '美食', 'environment': 0.0, 'price': 17.0, 'address': '翔殷路1099号合生汇4层', 'star': '四星商户', 'name': '阿满鸡排(合生汇店)', 'quality': 0.9, 'service': 0.7}, {'comment_count': 988, 'classify': '美食', 'environment': 0.4, 'price': 150.0, 'address': '淞沪路8号百联又一城购物中心7层', 'star': '四星商户', 'name': '莆田餐厅(百联又一城购物中心店)', 'quality': 0.0, 'service': 0.0}, {'comment_count': 152, 'classify': '美食', 'environment': 0.1, 'price': 16.0, 'address': '淞沪路8号百联又一城购物中心B1层', 'star': '准四星商户', 'name': '旬彩(百联又一城购物中心店)', 'quality': 0.2, 'service': 0.2}, {'comment_count': 72, 'classify': '美食', 'environment': 0.4, 'price': 21.0, 'address': '政旦东路34号', 'star': '准四星商户', 'name': '吴茂兴本帮面(五角场店)', 'quality': 0.4, 'service': 0.6}, {'comment_count': 3627, 'classify': '美食', 'environment': 0.5, 'price': 99.0, 'address': '翔殷路1099号合生汇5F', 'star': '四星商户', 'name': '雀舞云南菜(合生汇店)', 'quality': 0.0, 'service': 0.6}, {'comment_count': 336, 'classify': '美食', 'environment': 0.6, 'price': 79.0, 'address': '黄兴路2228号合生汇B2层城市集市C8商户', 'star': '准四星商户', 'name': '阳光客(城市集市合生汇店)', 'quality': 0.1, 'service': 0.7}, {'comment_count': 264, 'classify': '美食', 'environment': 0.7, 'price': 48.0, 'address': '翔殷路1099号合生汇B2城市集市C9', 'star': '四星商户', 'name': '越打星YuetDaSing(合生汇店)', 'quality': 0.0, 'service': 0.8}]\n",
      "总共加载15条数据\n"
     ]
    }
   ],
   "source": [
    "# 数据清洗\n",
    "\n",
    "datalst = []  # 创建空列表\n",
    "\n",
    "f.seek(0)\n",
    "n = 0  # 创建计数变量\n",
    "for i in f.readlines()[1:20]:\n",
    "    data = i.split(',')\n",
    "    #print(data)\n",
    "    classify = data[0]             # 提取分类\n",
    "    name = data[1]                 # 提取店铺名称\n",
    "    comment_count = fcm(data[2])   # 提取评论数量\n",
    "    star = data[3]                 # 提取星级\n",
    "    price = fpr(data[4])           # 提取人均\n",
    "    add = data[5]                  # 提取地址\n",
    "    qua = fcl(data[6])[0]          # 提取质量评分\n",
    "    env = fcl(data[6])[1]          # 提取环境评分\n",
    "    ser = fcl(data[6])[2]          # 提取服务评分\n",
    "    if  '缺失数据' not in [comment_count, price, qua]:   # 用于判断是否有数据缺失\n",
    "        n += 1\n",
    "        data_re = [['classify',classify],\n",
    "                  ['name',name],\n",
    "                  ['comment_count',comment_count],\n",
    "                  ['star',star],\n",
    "                  ['price',price],\n",
    "                  ['address',add],\n",
    "                  ['quality',qua],\n",
    "                  ['environment',env],\n",
    "                  ['service',ser]]\n",
    "        datalst.append(dict(data_re))   # 生成字典，并存入列表datalst\n",
    "        print('成功加载%i条数据' %n)\n",
    "    else:\n",
    "        continue\n",
    "    \n",
    "print(datalst) \n",
    "print('总共加载%i条数据' %n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finished!\n"
     ]
    }
   ],
   "source": [
    "# 数据存储.pkl数据\n",
    "\n",
    "import pickle\n",
    "pic = open('C:/Users/Hjx/Desktop/data.pkl','wb')\n",
    "pickle.dump(datalst,pic)\n",
    "pic.close()\n",
    "print('finished!')\n",
    "# 将数据存成了pkl文件"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
